library(jsonlite)

library(plyr) # rbind.fill is useful for ragged data with missing entries
library(dplyr)  
library(magrittr)
library(tibble)
library(stringr) 

library(sf)
library(tmap)
library(tmaptools) # for geocode_OSM

df_colours <- NULL
for (group in colour_groups) {
  source <- str_c(base_url, group) # the URL of the colour group JSON
  json <- fromJSON(source, flatten = TRUE)
  write_json(json, str_c(group, ".json"))
  # the next line needed a bit of experimentation...
  the_colours <- rbind.fill(json$categoryColours$masterColour.colours)
  if (is.null(df_colours)) {
    df_colours <- the_colours
  } else {
    df_colours <- bind_rows(df_colours, the_colours)
  }
  Sys.sleep(0.5)  
}
write.csv(df_colours, "dulux-colours-raw.csv", row.names = FALSE)

df_colours <- read.csv("dulux-colours-raw.csv")
head(df_colours)

df_colours_tidied <- df_colours %>%
  ## remove some columns we won't be needing
  select(-id, -baseId, -woodType, -coats) %>%
  ## separate the name components, filling from the left with NAs if <5
  separate(name, into = c("p1", "p2", "p3", "p4", "p5"), sep = " ", 
           remove = FALSE, fill = "left") %>%
  ## replace any NAs with an empty string
  mutate(p1 = str_replace_na(p1, ""),
         p2 = str_replace_na(p2, ""),
         p3 = str_replace_na(p3, ""),
         p4 = str_replace_na(p4, "")) %>%
  ## if p5 is a paint modifiers, then recompose name 
  ## from p1:p4 else from p1:p5
  ## similarly keep modifier where it exists
  mutate(placename = if_else(p5 %in% paint_modifiers, 
                       str_trim(str_c(p1, p2, p3, p4, sep = " ")), 
                       str_trim(str_c(p1, p2, p3, p4, p5, sep = " "))),
         modifier = if_else(p5 %in% paint_modifiers, 
                       p5, "")) %>%
  ## remove some places that are tricky to deal with later
  filter(!placename %in% c("Chatham Islands", 
                           "Passage Rock", 
                           "Auckland Islands", 
                           "Cossack Rock")) %>%
  ## throw away variables we no longer and reorder
  select(name, placename, modifier, red, green, blue)

# save it so we have it for later
write.csv(df_colours_tidied, "dulux-colours.csv", row.names = FALSE)

for (placename in unique(df_colours_tidied_xy$placename)) {
  address <- str_c(placename, "New Zealand", sep = ", ")
  geocode <- geocode_OSM(address, as.data.frame = TRUE, return.first.only = FALSE)
  num_geocodes <- nrow(geocode)
  matching_rows <- which(df_colours_tidied_xy$placename == placename)
  for (i in 1:length(matching_rows)) {
    if (!is.null(geocode)) {
      if (num_geocodes >= i) {
        df_colours_tidied_xy[matching_rows[i], ]$x <- geocode$lon[i] 
        df_colours_tidied_xy[matching_rows[i], ]$y <- geocode$lat[i] 
      }  
    }
  }
  Sys.sleep(0.5) # so as not to over-tax the geocoder
}

df_colours_tidied_xy <- df_colours_tidied_xy %>%
  filter(x != 0 & y != 0)

write.csv(df_colours_tidied_xy, "dulux-colours-xy.csv", row.names = FALSE)

dulux_colours_sf <- st_as_sf(dulux_colours, 
                     coords = c("x", "y"), # the columns with the coordinates
                     crs = 4326) %>%       # the project EPSG:4326 for lng-lat
  st_transform(2193) %>% # convert to NZTM
  ## and make an RGB column
  mutate(rgb = rgb(red / 255, 
                   green / 255, 
                   blue/ 255))

st_write(dulux_colours_sf, "dulux-colours-pts.gpkg", delete_dsn = TRUE)

tmap_mode("view")

tm_shape(st_read("nz.gpkg")) +
  tm_borders() +
  tm_shape(dulux_colours_sf) + 
  tm_dots(col = "rgb")

dulux_colours_vor <- st_read("dulux-colours-vor.gpkg")

tm_shape(dulux_colours_vor) + 
  tm_polygons(col = "rgb", id = "placename", alpha = 0.75, border.col = "grey", lwd = 0.2) +
  tm_basemap("Esri.WorldTopoMap")